package org.zsen.crawler;
import java.util.regex.Pattern;

import org.jsoup.helper.StringUtil;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import cn.edu.hfut.dmic.webcollector.crawler.BreadthCrawler;
import cn.edu.hfut.dmic.webcollector.model.Links;
import cn.edu.hfut.dmic.webcollector.model.Page;

/**
 * Crawl news from yahoo news
 *
 * @author hu
 */
public class YahooCrawler extends BreadthCrawler {

	public static final Logger LOG = LoggerFactory.getLogger(YahooCrawler.class);
    /**
     * @param crawlPath crawlPath is the path of the directory which maintains
     * information of this crawler
     * @param autoParse if autoParse is true,BreadthCrawler will auto extract
     * links which match regex rules from pag
     */
    public YahooCrawler(String crawlPath, boolean autoParse) {
        super(crawlPath, autoParse);
//        /*start page*/
//        this.addSeed("http://news.yahoo.com/");
//
//        /*fetch url like http://news.yahoo.com/xxxxx*/
//        this.addRegex("http://news.yahoo.com/.*");
//        /*do not fetch url like http://news.yahoo.com/xxxx/xxx)*/
//        this.addRegex("-http://news.yahoo.com/.+/.*");
//        /*do not fetch jpg|png|gif*/
//        this.addRegex("-.*\\.(jpg|png|gif).*");
//        /*do not fetch url contains #*/
//        this.addRegex("-.*#.*");
        this.addSeed("http://top.baidu.com/");
        this.addRegex("http://top.baidu.com/buzz.*");
    }

    @Override
    public void visit(Page page, Links nextLinks) {
        String url = page.getUrl();
        /*if page is news page*/
        if (Pattern.matches("http://top.baidu.com/buzz.*", url)) {
            /*we use jsoup to parse page*/
            Document doc = page.getDoc();
				LOG.info(doc.select("title")+"  "+url);
				LOG.info("############Start#######");
				for(Element e:doc.select("tbody").select("tr"))
				{
					if(!StringUtil.isBlank(e.select("td.keyword").text()))
					LOG.info("["+e.select("td.keyword").select("a.list-title").text()+"   "+e.select("td.last").select("span").text()+"]");
				}
				
				LOG.info("############End#######");
		
            /*extract title and content of news by css selector*/
//            String title = doc.select("h1[class=headline]").first().text();
//            String content = doc.select("div[class=body yom-art-content clearfix]").first().text();
//
//            System.out.println("URL:\n" + url);
//            System.out.println("title:\n" + title);
//            System.out.println("content:\n" + content);

            /*If you want to add urls to crawl,add them to nextLink*/
            /*WebCollector automatically filters links that have been fetched before*/
            /*If autoParse is true and the link you add to nextLinks does not match the regex rules,the link will also been filtered.*/
            // nextLinks.add("http://xxxxxx.com");
        }

    }

    public static void main(String[] args) throws Exception {
        YahooCrawler crawler = new YahooCrawler("crawl", true);
        crawler.setThreads(5);
        crawler.setTopN(1000);
        crawler.setResumable(false);
        //crawler.setResumable(true);
        crawler.start(10);
        
/*        String url = "http://www.huxiu.com/article/121959/1.html";
        News news = ContentExtractor.getNewsByUrl(url);
        System.out.println(news);
        String content = ContentExtractor.getContentByUrl(url);
        System.err.println(content);*/


    }

}
